package com.seo.textgen.tokenizer;

import java.util.ArrayList;

import com.seo.textgen.classifier.CharClassifier;
import com.seo.textgen.tokenizer.filter.DefaultTokenizerSentenceFilter;
import com.seo.textgen.tokenizer.filter.TokenizerSentenceFilter;

public class Tokenizer {

	private CharClassifier charClassifier = new CharClassifier();
	
	private TokenizerSentenceFilter tokenizerSentenceFilter = new DefaultTokenizerSentenceFilter();

	public void tokenize(String text, TokenizerCallback callback) {
		text = preprocessText(text);
		
		ArrayList<Token> sentence = new ArrayList<Token>();
		Token lastToken = null;

		Integer wordStart = null;
		for (int i = 0; i < text.length(); i++) {
			char chr = text.charAt(i);

			if (charClassifier.isWordStart(chr)) {
				// Start word token
				if (wordStart == null) {
					wordStart = i;
				}
				continue;
			}

			if (charClassifier.isWordChar(chr)) {
				// Continue word token
				if (wordStart != null) {
					continue;
				}
			} else {
				// End of word token
				if (wordStart != null) {
					// Skip word punkt chars in the end
					if (charClassifier.isWordPunkt(text.charAt(i - 1))) {
						do {
							i--;
						} while (charClassifier.isWordPunkt(text.charAt(i - 1)));
						chr = text.charAt(i);
					}
					String token = text.substring(wordStart, i);
					lastToken = new Token(TokenType.WORD, token);
					sentence.add(lastToken);
					wordStart = null;
				}
			}

			if (charClassifier.isPunkt(chr)) {
				lastToken = new Token(TokenType.PUNKT, String.valueOf(chr));
				sentence.add(lastToken);
				continue;
			}

			if (charClassifier.isEndOfSentence(chr) || charClassifier.isNewline(chr)) {
				if (charClassifier.isNewline(chr)) {
					lastToken = new Token(TokenType.END_OF_SENTENCE, ".");
				} else {
					lastToken = new Token(TokenType.END_OF_SENTENCE, String.valueOf(chr));
				}
				sentence.add(lastToken);
				
				while (++i < text.length() && (charClassifier.isEndOfSentence(text.charAt(i)) || charClassifier.isNewline(text.charAt(i))));
				
				if (tokenizerSentenceFilter.filter(sentence)) {
					callback.processSentence(sentence.toArray(new Token[sentence.size()]));
				}
				sentence.clear();
			}
		}
	}

	private String preprocessText(String text) {
		char[] chars = new char[text.length()];
		
		int p = 0;
		for (int i = 0; i < text.length(); i++) {
			if (!charClassifier.isOther(text.charAt(i))) {
				chars[p++] = text.charAt(i);
			}
		}
		return new String(chars, 0, p);
	}
}
